# coding=utf-8

import queue
import threading
from time import sleep
import os

# login
from selenium import webdriver

# scroll2bottom
from selenium.webdriver.common.action_chains import ActionChains
    
# each_down
import requests
from bs4 import BeautifulSoup 
    
base_url = "http://friend.renren.com/managefriends"


class List:

    def __init__(self, url, title):
        self.url = url
        self.title = title

        
class Each:

    def __init__(self, url, path):
        self.url = url
        self.path = path


class Statistics:
    
    def __init__(self):
        self.album = 0
        self.task = 0
        self.done = 0
        self.lock = threading.Lock()
        
    def __str__(self):
        return "album=%s task=%s done=%s" % (self.album, self.task, self.done)

    def add_album(self):
        self.lock.acquire()  # 加锁，锁住相应的资源
        self.album += 1
        self.lock.release()  # 解锁，离开该资源
        
    def add_task(self):
        self.lock.acquire()  # 加锁，锁住相应的资源
        self.task += 1
        self.lock.release()  # 解锁，离开该资源
        
    def add_done(self):
        self.lock.acquire()  # 加锁，锁住相应的资源
        self.done += 1
        self.lock.release()  # 解锁，离开该资源


q_each = queue.Queue(maxsize=50)
statistics = Statistics()
driver = None
flag_run = True


def p(str1):
    if 1:
        print("%s" % str1)

    
def list_2_each(lists, q):
    global statistics
    
    for li_obj in lists:
        a_obj = li_obj.find_element_by_tag_name("a")
        if a_obj:
            a_link = a_obj.get_attribute('href') 
#             p("PUT %s" % a_link)
            q.put(a_link)
            statistics.add_task()
        else:
            print("error: no a_link")
            continue
        
    print("info: list_2_each OVER")


def each_down(cookies, q):

    global statistics
    
    s = requests.Session()
    for item in cookies:
        s.cookies.set(item["name"], item["value"])
    
    while flag_run:
        try:
            each1 = q.get(timeout=30)
        except:
            print("info: each_down 30s break")
            break
        
        p("GET:%s" % each1)
        for i in range(0, 3):
            try:
                response = s.get(each1)
#                 sleep(5)
                
                content = response.content.decode("utf-8", "ignore")
                soup = BeautifulSoup(content, "html.parser")
                visit_obj = soup.select('div.cover-bg h1')
                
                if visit_obj:
                    p(visit_obj)
                    statistics.add_done()
                else:
                    os.system("pause")
                
#                 sleep(5)
                p("DONE %s" % each1)
                break
            except:
                if i < 3:
                    sleep(1)
                else:
                    print("error: get(%s)", each1)
                    
    print("info: each_down OVER")

    
def login():
    
    global driver
    
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument(r"user-data-dir=C:\Users\Administrator\AppData\Local\Google\Chrome\User Data")
    # chrome_options.add_argument('--headless')
    # chrome_options.add_argument('--disable-gpu')
    # prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': 'I:\\'}
    # chrome_options.add_experimental_option('prefs', prefs)
    driver = webdriver.Chrome(executable_path=r"D:\Python\Python37\Scripts\chromedriver.exe", chrome_options=chrome_options)
    
    driver.get(base_url)
    
    for i in range(0, 3):
        try:
            sleep(1)
#             p(driver.page_source)
            driver.page_source.index("webpagerengine")
            return True, driver.get_cookies()
        except:
            if i < 3:
                try:
                    driver.find_element_by_id("login").click()
                except:
                    print("error: login click")
            else:
                print("error: login 3 times")

    return False


def scroll2bottom():
    
    global driver

    total_obj = driver.find_element_by_id('f_group-num')
    if total_obj:
        total = int(total_obj.text)
        p("total=%s" % total)
    else:
        print("error: not found total_obj")
        return False
    
    while flag_run:
        lists = driver.find_elements_by_class_name('friend-detail')
        count = len(lists)
        p("count=%s" % count)
        
        if count == total:
            return True, lists
        else:
            if count != 0:
#                 print(lists[-1].text)
                ActionChains(driver).move_to_element(lists[-1]).perform()
                sleep(0.5)
                continue
            else:
                print("error: count=0")
                return False

    
if __name__ == '__main__':
    
    ret, cookies = login()
    if not ret:
        print("error: login")
        os.system("pause")
        exit()
    
    ret, lists = scroll2bottom()
    if not ret:
        print("error: scroll2bottom")
        os.system("pause")
        exit()
    
    # 下载图片
    threads = []
     
    # 2、有效页->每张单独页
    for i in range(0, 1):
        t = threading.Thread(target=list_2_each, args=(lists, q_each))
        threads.append(t)
    
    # 3、每张单独页下载
    for i in range(0, 10):
        t = threading.Thread(target=each_down, args=(cookies, q_each))
        threads.append(t)

    # 开干
    count = range(len(threads))
    for i in count:
        threads[i].start()

    i = 0
    while True:
        sleep(5)
        print("info: q_each=%s %s" % (q_each.qsize(), statistics))
        if q_each.qsize() == 0:
            if i < 3:
                i += 1
            else:
                break
         
    # 收工
    flag_run = False
    for i in count:
        threads[i].join()
    print("*** OVER ***")
